﻿Imports System.Net
Imports System.IO
Imports Google.GData.Client
Imports Google.GData.Extensions
Imports Google.GData.Spreadsheets
Namespace IGNOU



    Public Class StudentAddressParser
        Inherits Student
        Public Enum StudentStatus
            None = 1
            Pending = 2
            EnrollmentNoNotFound = 3
            NetworkError = 4
            Other = 5
            Success = 6
            DataFoundButNotParsed = 7
        End Enum

        Property Data As String

        Property Status As StudentStatus

        Sub ParseIt()
            Me.Status = StudentStatus.Success
            Me.EnrollmentNo = GetElementValue("Enrollment No.")
            Me.Program = GetElementValue("Program")
            Me.FullName = GetElementValue("Name")
            Me.Guardian = GetElementValue("Father/Husband/Mother's Name")
            Me.Address = GetElementValue("Current Address")
            Try
                Me.Address = Regex.Replace(Me.Address, "<br>", vbCrLf)
            Catch ex As Exception

            End Try

            Me.RegionalCenterCode = GetElementValue("Reg.Centre Code")
            Me.StudyCenterCode = GetElementValue("Study Code")
            Me.Mobile = GetElementValue("Mobile No.")
            Me.EmailID = GetElementValue("Email-ID")
        End Sub

        Private Function GetElementValue(ByVal s As String) As String
            Dim rx As Match

            rx = Regex.Match(Me.Data, "(?<=<td>" & s & "</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            If rx.Success Then
                Return rx.Value
            Else
                Me.Status = StudentStatus.DataFoundButNotParsed
                Return ""
            End If
        End Function

    End Class

    Public Class Student
        Property EnrollmentNo As String
        Property Program As String
        Property FullName As String
        Property Guardian As String
        Property Address As String
        Property RegionalCenterCode As String
        Property StudyCenterCode As String
        Property Mobile As String
        Property EmailID As String
        '    Enrollment No.	063072609
        'Program	MCA
        'Name	MANISH RAWAT
        'Father/Husband/Mother's Name	VIRENDER SINGH RAWAT
        'Current Address	C-362 POPCKET-1 SECTOR-17
        'ROHINI

        '-110085
        'NEW DELHI
        'Reg.Centre Code	29
        'Study Code	0731
        'Mobile No.	
        'Email-ID	
    End Class

    Public Class IGNOURobot

        Shared Function ParseEnrollmentNumbers(ByVal s As String) As List(Of String)

            Dim rx = System.Text.RegularExpressions.Regex.Matches(s, "\d{9}")

            Dim st = (From t As RegularExpressions.Match In rx Select t.Value).ToList

            Return st

        End Function

        Public Shared prxy As New System.Net.WebProxy("119.235.54.122", 3128)

        Shared Function GetStudent(ByVal e As String) As IGNOU.StudentAddressParser
            Return GetStudent(e, True)
        End Function

        Shared Function GetStudent(ByVal e As String, ByVal doParse As Boolean) As IGNOU.StudentAddressParser
            Dim o As New System.Collections.Specialized.NameValueCollection
            Dim st As New StudentAddressParser

            o.Add("EnrNo", e)
            o.Add("Submit", "Submit")
            o.Add("program", "' OR enrno='" & e)

            Try
                Dim r = GetRequest("http://admission.ignou.ac.in/changeadmdata/ADDRESSSTATUS.ASP", o)

                Dim resp = r.GetResponse.GetResponseStream
                Dim reader As New System.IO.StreamReader(resp)
                Dim t = reader.ReadToEnd

                st.Data = t
                If doParse Then
                    If Regex.IsMatch(t, "Enrollment No.063072609 Not found...") Then
                        st.Status = StudentAddressParser.StudentStatus.EnrollmentNoNotFound
                    Else
                        st.ParseIt()
                    End If
                End If

            Catch ex As System.IO.IOException
                st.Status = StudentAddressParser.StudentStatus.NetworkError
            Catch ex As Exception
                st.Status = StudentAddressParser.StudentStatus.Other
            End Try


            Return st

            'Dim rx As Match

            'rx = Regex.Match(t, "(?<=<td>Enrollment No.</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            'If rx.Success Then
            '    st.EnrollmentNo = rx.Value
            'End If

            'rx = Regex.Match(t, "(?<=<td>Program</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            'If rx.Success Then
            '    st.Program = rx.Value
            'End If

            'rx = Regex.Match(t, "(?<=<td>Name</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            'If rx.Success Then
            '    st.FullName = rx.Value
            'End If


            'rx = Regex.Match(t, "(?<=<td>Father/Husband/Mother's Name</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            'If rx.Success Then
            '    st.Guardian = rx.Value
            'End If


            'rx = Regex.Match(t, "(?<=<td>Current Address</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            'If rx.Success Then
            '    st.Address = rx.Value
            'End If


            'rx = Regex.Match(t, "(?<=<td>Reg.Centre Code</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            'If rx.Success Then
            '    st.RegionalCenterCode = rx.Value
            'End If


            'rx = Regex.Match(t, "(?<=<td>Study Code</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            'If rx.Success Then
            '    st.StudyCenterCode = rx.Value
            'End If


            'rx = Regex.Match(t, "(?<=<td>Mobile No.</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            'If rx.Success Then
            '    st.Mobile = rx.Value
            'End If

            'rx = Regex.Match(t, "(?<=<td>Email-ID</td><td>).*?(?=</td>)", RegexOptions.IgnoreCase)

            'If rx.Success Then
            '    st.EmailID = rx.Value
            'End If

            'Return st

            't = Regex.Replace(t, "t", "")


            '<tr class=(td2|td1).*?</tr>


            'Dim rx = Regex.Matches(t, "<tr class=(td2|td1).*?</tr>", RegexOptions.IgnoreCase)


            'For Each m As Match In rx

            '    Dim tdColl = m.Value
            '    Dim againRx = Regex.Matches(tdColl, "<td.*?</td>", RegexOptions.IgnoreCase)

            '    If againRx.Count = 2 Then



            '    End If

            'Next


            'Dim p = Server.MapPath("~/123.txt")
            'My.Computer.FileSystem.WriteAllText(p, t, True)
            'Response.Write(t)
        End Function

        Shared Function GetRequest(ByVal url As [String], ByVal nameValueCollection As Specialized.NameValueCollection) As HttpWebRequest
            ' Here we convert the nameValueCollection to POST data.
            ' This will only work if nameValueCollection contains some items.
            Dim parameters = New StringBuilder()

            For Each key In nameValueCollection
                parameters.AppendFormat("{0}={1}&", HttpUtility.UrlEncode(key), HttpUtility.UrlEncode(nameValueCollection(key)))
            Next

            parameters.Length -= 1

            ' Here we create the request and write the POST data to it.
            Dim request = DirectCast(HttpWebRequest.Create(url), HttpWebRequest)
            request.ContentType = "application/x-www-form-urlencoded"
            request.Method = "POST"
            request.AutomaticDecompression = System.Net.DecompressionMethods.GZip
            request.Proxy = prxy 'New System.Net.WebProxy("119.82.113.131", 3128)
            'Accept-Encoding: gzip,deflate
            Using writer = New StreamWriter(request.GetRequestStream())
                writer.Write(parameters.ToString())
            End Using

            Return request
        End Function
    End Class

    Public Class GenericParser
        Shared Function Parse(Of t As New, p As IParser(Of t))(ByVal d As String) As t
            Dim o As New t

        End Function
    End Class

    Public Interface IParser(Of T)
        Function Parse(ByVal d As String) As ParseResult(Of T)
    End Interface

    Public Class ParseResult(Of t)
        Property Data As String
        Property Result As t
    End Class

    Public Class StudentAddressCollectorHandler
        'Inherits List(Of String)

        Dim rst As New System.Threading.AutoResetEvent(True)

        Property IsRunning As Boolean
        Dim _pathToSave, _PendingFolderPath, _CompletedFolderPath As String

        ReadOnly Property ExistsInPendingFolder As Boolean
            Get

            End Get
        End Property

        Sub AddToCompletedFolder(ByVal enr As String)
            Dim fs As New System.IO.DirectoryInfo(CompletedFolderPath)
            Dim dToSave = (From t In fs.GetDirectories Where t.GetFiles.Count < 1000).FirstOrDefault
            If dToSave Is Nothing Then
                dToSave = fs.CreateSubdirectory(Now.Ticks)
            End If

            If dToSave Is Nothing Then
                dToSave = fs
            End If

            Dim fileToSave = dToSave.FullName & "\" & enr & ".txt"
            System.IO.File.Create(fileToSave).Close()
        End Sub

        Sub AddToQueueFolder(ByVal enr As String)
            Dim fs As New System.IO.DirectoryInfo(PendingFolderPath)
            Dim dToSave = (From t In fs.GetDirectories Where t.GetFiles.Count < 1000).FirstOrDefault
            If dToSave Is Nothing Then
                dToSave = fs.CreateSubdirectory(Now.Ticks)
            End If

            If dToSave Is Nothing Then
                dToSave = fs
            End If

            Dim fileToSave = dToSave.FullName & "\" & enr & ".txt"
            System.IO.File.Create(fileToSave).Close()
        End Sub

        Sub SaveToSaveFolder(ByVal enr As String, ByVal obj As IGNOU.StudentAddressParser)
            Dim fs As New System.IO.DirectoryInfo(PathToSave)
            Dim dToSave = (From t In fs.GetDirectories Where t.GetFiles.Count < 1000).FirstOrDefault
            If dToSave Is Nothing Then
                dToSave = fs.CreateSubdirectory(Now.Ticks)
            End If

            If dToSave Is Nothing Then
                dToSave = fs
            End If

            Dim fileToSave = dToSave.FullName & "\" & enr & ".txt"

            'Dim f=From t In My.Computer.FileSystem.GetDirectories(PathToSave) Where t
            My.Computer.FileSystem.WriteAllText(fileToSave, obj.Data, False)

        End Sub

        ReadOnly Property NeedToQueue(ByVal enr As String) As Boolean
            Get
                Dim b As Boolean
                Try
                    Dim fs As System.IO.DirectoryInfo
                    fs = New System.IO.DirectoryInfo(PendingFolderPath)
                    If (From t In fs.GetFiles(enr & ".txt", SearchOption.AllDirectories)).Count > 0 Then
                        Return False
                    End If

                    fs = New System.IO.DirectoryInfo(CompletedFolderPath)
                    If (From t In fs.GetFiles(enr & ".txt", SearchOption.AllDirectories)).Count > 0 Then
                        Return False
                    End If

                    Return True

                Catch ex As Exception

                End Try
                Return b

            End Get
        End Property

        Sub Add(ByVal enrollmentNo As String)

            If NeedToQueue(enrollmentNo) Then

                AddToQueueFolder(enrollmentNo)

                rst.Set()

                IsRunning = True

            End If


            'If Not My.Computer.FileSystem.FileExists(PendingFolderPath & enrollmentNo & ".txt") AndAlso _
            '    Not My.Computer.FileSystem.FileExists(CompletedFolderPath & enrollmentNo & ".txt") Then

            '    'Dim f As New System.IO.File
            '    System.IO.File.Create(PendingFolderPath & enrollmentNo & ".txt").Close()


            '    'My.Computer.FileSystem.WriteAllText(PendingFolderPath & enrollmentNo & ".txt", "", False)

            '    rst.Set()

            '    IsRunning = True

            'End If
        End Sub

        ReadOnly Property PathToSave As String
            Get
                Return _pathToSave
            End Get
        End Property

        ReadOnly Property PendingFolderPath As String
            Get
                Return _PendingFolderPath
            End Get
        End Property

        ReadOnly Property CompletedFolderPath As String
            Get
                Return _CompletedFolderPath
            End Get
        End Property

        Sub New()
            _pathToSave = HttpContext.Current.Server.MapPath("~/Ignou/Data/")
            _PendingFolderPath = HttpContext.Current.Server.MapPath("~/Ignou/Pending/")
            _CompletedFolderPath = HttpContext.Current.Server.MapPath("~/Ignou/Completed/")

            Dim th As New Threading.Thread(AddressOf StartScraping)
            th.Start()

            Dim th1 As New Threading.Thread(AddressOf StartConversionGoogleData)
            th1.Start()

        End Sub



        'Sub AddToList(ByVal s As String)
        '    If Not My.Computer.FileSystem.FileExists(PathToSave & s & ".txt") AndAlso Not Me.Contains(s) Then
        '        Me.Add(s)
        '        rst.Set()
        '        IsRunning = True
        '    End If
        'End Sub

        Private Sub StartConversionGoogleData()
            Dim fs As New System.IO.DirectoryInfo(PathToSave)
            Dim files = From t In fs.GetFiles("*.txt", SearchOption.AllDirectories)

            For Each f In files
                Try
                    Dim obj As New StudentAddressParser
                    obj.Data = f.OpenText.ReadToEnd
                    obj.ParseIt()
                    If obj.Status = StudentAddressParser.StudentStatus.Success Then
                        Dim result = StudentGoogleDAL.Insert(obj)
                        If result.OperationStatus Then
                            f.Delete()
                        End If
                    End If
                    
                Catch ex As Exception

                End Try
                
            Next


        End Sub

        Private Sub StartScraping()
            Do

                'Dim fileToLoad = (From t In My.Computer.FileSystem.GetFiles(PendingFolderPath) Select t).FirstOrDefault
                Try
                    Dim fs As New System.IO.DirectoryInfo(PendingFolderPath)

                    Dim ckck = (From t In fs.GetFiles("*.txt", SearchOption.AllDirectories) Select t).FirstOrDefault

                    If ckck IsNot Nothing Then
                        Dim t = ckck.Name
                        Dim enr = ckck.Name.Replace(ckck.Extension, "")
                        Dim o = IGNOURobot.GetStudent(enr, False)
                        Dim f = PathToSave & ckck.Name
                        SaveToSaveFolder(enr, o)
                        'My.Computer.FileSystem.WriteAllText(f, o.Data, False)
                        'Me.RemoveAt(0)
                        'My.Computer.FileSystem.m
                        'ckck.MoveTo(CompletedFolderPath & t)
                        ckck.Delete()

                        AddToCompletedFolder(enr)
                        IsRunning = True
                    Else

                        IsRunning = False
                        rst.WaitOne()

                    End If
                Catch ex As Exception

                End Try

            Loop
        End Sub

        ReadOnly Property TotalProcessed As Integer
            Get

                Dim fs As New System.IO.DirectoryInfo(CompletedFolderPath)
                Dim cnt = fs.GetFiles("*.txt", SearchOption.AllDirectories).Count
                Return cnt
                'Return My.Computer.FileSystem.GetFiles(CompletedFolderPath, FileIO.SearchOption.SearchAllSubDirectories).Count
            End Get
        End Property

        ReadOnly Property TotalPending As Integer
            Get

                Dim fs As New System.IO.DirectoryInfo(PendingFolderPath)
                Dim cnt = fs.GetFiles("*.txt", SearchOption.AllDirectories).Count
                Return cnt

                'Return My.Computer.FileSystem.GetFiles(PendingFolderPath, FileIO.SearchOption.SearchAllSubDirectories).Count
            End Get
        End Property

    End Class

    Public Class StudentGoogleDAL

        Enum ColumnPosition
            None = 0
            EnrollmentNo = 1
            FullName = 2
            Address = 3
            Email = 4
            Phone = 5
            Guardian = 6
            Program = 7
            SCCode = 8
            RCCode = 9
        End Enum

        Shared _spreadsheetSVC As SpreadsheetsService = Nothing
        Shared _lfFeed As ListFeed = Nothing
        Const UserName As String = "mnsrulz@gmail.com", Pwd As String = "9899888450"
        Const CellQueryLink = "https://spreadsheets.google.com/feeds/cells/0AlnO9Mbz781SdEkyX3ZpVFdJOU9EbndkZGVYMHlFZ1E/od6/private/full"
        Const ListQueryLink = "https://spreadsheets.google.com/feeds/list/tI2_viTWI9ODnwddeX0yEgQ/od6/private/full"

        Shared Function GetSpreadsheetService() As SpreadsheetsService
            If _spreadsheetSVC Is Nothing Then
                Dim spreadsheetSVC As New SpreadsheetsService("ignou-scraper")
                With spreadsheetSVC
                    '.Credentials = New Google.GData.Client.GDataCredentials(UserName, Pwd)
                End With


                Dim f As GDataRequestFactory = spreadsheetSVC.RequestFactory
                f.UseSSL = False
                f.Proxy = New System.Net.WebProxy("http://google.com", 80)
                'f.Proxy = IGNOURobot.prxy
                _spreadsheetSVC = spreadsheetSVC

            End If
            Return _spreadsheetSVC
        End Function

        Sub New()

        End Sub

        Shared Function GetListFeed() As ListFeed
            If _lfFeed Is Nothing Then
                Dim service = GetSpreadsheetService()
                Dim LFquery = New ListQuery(ListQueryLink)
                LFquery.NumberToRetrieve = 1
                _lfFeed = service.Query(LFquery)
            End If
            Return _lfFeed
        End Function

        Shared Function ListAll(ByVal startRow As Integer, ByVal maximumRows As Integer) As BOResult(Of List(Of Student))
            Dim objResult As New BOResult(Of List(Of Student))
            Try
                Dim service = GetSpreadsheetService() ' New SpreadsheetsService("exampleCo-exampleApp-1")
                'service.setUserCredentials(UserName, Pwd)
                Dim q = New CellQuery(CellQueryLink)
                With q
                    .MinimumColumn = 1
                    .MaximumColumn = 10
                    .MinimumRow = startRow + 1  'exclude the first row as it contains the header column names
                    .MaximumRow = startRow + maximumRows - 1
                End With

                Dim cellFeed = service.Query(q)

                If cellFeed IsNot Nothing AndAlso cellFeed.Entries IsNot Nothing Then
                    Dim totalRows = (From t As CellEntry In cellFeed.Entries Select t.Row)

                    Dim res = (From t As CellEntry In cellFeed.Entries Group By t.Row Into Group Select Group, Row).ToList

                    Dim lst As New List(Of Student)
                    objResult.Result = lst
                    objResult.TotalRecords = cellFeed.RowCount.Count
                    For i As Integer = 0 To res.Count - 1
                        Dim obj As New Student
                        obj.FullName = (From t In res.Item(i).Group Where t.Column = ColumnPosition.FullName Select t.Value).FirstOrDefault
                        obj.EnrollmentNo = (From t In res.Item(i).Group Where t.Column = ColumnPosition.EnrollmentNo Select t.Value).FirstOrDefault
                        obj.EmailID = (From t In res.Item(i).Group Where t.Column = ColumnPosition.Email Select t.Value).FirstOrDefault
                        obj.Mobile = (From t In res.Item(i).Group Where t.Column = ColumnPosition.Phone Select t.Value).FirstOrDefault
                        obj.Guardian = (From t In res.Item(i).Group Where t.Column = ColumnPosition.Guardian Select t.Value).FirstOrDefault
                        obj.Address = (From t In res.Item(i).Group Where t.Column = ColumnPosition.Address Select t.Value).FirstOrDefault
                        obj.Program = (From t In res.Item(i).Group Where t.Column = ColumnPosition.Program Select t.Value).FirstOrDefault
                        obj.StudyCenterCode = (From t In res.Item(i).Group Where t.Column = ColumnPosition.SCCode Select t.Value).FirstOrDefault
                        obj.RegionalCenterCode = (From t In res.Item(i).Group Where t.Column = ColumnPosition.RCCode Select t.Value).FirstOrDefault

                        lst.Add(obj)
                    Next
                Else
                    Throw New Exception("No Rows Returned")
                End If
                objResult.OperationStatus = True
            Catch ex As Exception
                objResult.OperationStatus = False
                objResult.Status = "An Unknown Error occurred."
            End Try

            Return objResult
        End Function

        Private Shared Function BuildElement(ByVal localName As String, ByVal value As String) As ListEntry.Custom
            Dim curElement = New ListEntry.Custom
            curElement.LocalName = localName
            curElement.Value = value
            Return curElement
        End Function

        Shared Function Insert(ByVal objStudent As Student) As BOResult(Of Student)
            Dim objResult As New BOResult(Of Student)
            Try
                'Dim service = _spreadsheetSVC
                'Dim LFquery = New ListQuery(ListQueryLink)
                'LFquery.NumberToRetrieve = 1
                'Dim LFfeed = service.Query(LFquery)
                Dim lfFeed = GetListFeed()
                Dim newRow = New ListEntry()

                'enrollment_no	full_name	address	email	mobile	guardian 
                newRow.Elements.Add(BuildElement("enrollment_no", objStudent.FullName))
                newRow.Elements.Add(BuildElement("full_name", objStudent.FullName))
                newRow.Elements.Add(BuildElement("address", objStudent.Address))
                newRow.Elements.Add(BuildElement("email", objStudent.EmailID))
                newRow.Elements.Add(BuildElement("mobile", objStudent.Mobile))
                newRow.Elements.Add(BuildElement("guardian", objStudent.Guardian))
                newRow.Elements.Add(BuildElement("program", objStudent.Program))
                newRow.Elements.Add(BuildElement("rc_code", objStudent.RegionalCenterCode))
                newRow.Elements.Add(BuildElement("sc_code", objStudent.StudyCenterCode))

                lfFeed.Insert(newRow)


            Catch ex As Exception
                objResult.OperationStatus = False
                objResult.Status = "An unknown Error occurred."
            End Try
            Return objResult
        End Function

        Shared Function Exists(ByVal e As String) As BOResult(Of Boolean)
            Dim objREsult = New BOResult(Of Boolean)
            Try
                Dim service = GetSpreadsheetService()
                Dim LFquery = New ListQuery(ListQueryLink)
                LFquery.NumberToRetrieve = 1
                LFquery.SpreadsheetQuery = "enrollment_no=" & e
                Dim feeds = service.Query(LFquery)
                If feeds.TotalResults > 0 Then
                    objREsult.OperationStatus = True
                    objREsult.Result = True
                Else
                    objREsult.OperationStatus = True
                    objREsult.Result = False
                End If
            Catch ex As Exception
                objREsult.OperationStatus = False

            End Try

            Return objREsult
        End Function
    End Class


End Namespace
